In [1]:

    

import pandas as pd
import pandas.rpy.common as com
import numpy as np

%load_ext autoreload
%autoreload 2

%load_ext rmagic


    

In [2]:

    

%%R
library(ndl)


    

    






This is ndl version 0.2.16. 
For an overview of the package, type 'help("ndl.package")'.

In [3]:

    

data = com.load_data('danks')
data


    

    
Out[3]:






  

    

      

      
Cues
      
Outcomes
      
Frequency
    
  
  

    

      
1
      
 pots_red_blue
      
 y
      
  5
    
    

      
2
      
 pots_red_blue
      
 n
      
  0
    
    

      
3
      
      pots_red
      
 y
      
 10
    
    

      
4
      
      pots_red
      
 n
      
  5
    
    

      
5
      
     pots_blue
      
 y
      
  5
    
    

      
6
      
     pots_blue
      
 n
      
 10
    
    

      
7
      
          pots
      
 y
      
  0
    
    

      
8
      
          pots
      
 n
      
  5
    
  

8 rows × 3 columns

In [4]:

    

marginals = data.groupby('Cues',as_index=False).Frequency.sum()
marginals = marginals.rename(columns={'Frequency':'Total'})
marginals


    

    
Out[4]:






  

    

      

      
Cues
      
Total
    
  
  

    

      
0
      
          pots
      
  5
    
    

      
1
      
     pots_blue
      
 15
    
    

      
2
      
      pots_red
      
 15
    
    

      
3
      
 pots_red_blue
      
  5
    
  

4 rows × 2 columns

In [5]:

    

data = pd.merge(data,marginals,on='Cues')
data


    

    
Out[5]:






  

    

      

      
Cues
      
Outcomes
      
Frequency
      
Total
    
  
  

    

      
0
      
 pots_red_blue
      
 y
      
  5
      
  5
    
    

      
1
      
 pots_red_blue
      
 n
      
  0
      
  5
    
    

      
2
      
      pots_red
      
 y
      
 10
      
 15
    
    

      
3
      
      pots_red
      
 n
      
  5
      
 15
    
    

      
4
      
     pots_blue
      
 y
      
  5
      
 15
    
    

      
5
      
     pots_blue
      
 n
      
 10
      
 15
    
    

      
6
      
          pots
      
 y
      
  0
      
  5
    
    

      
7
      
          pots
      
 n
      
  5
      
  5
    
  

8 rows × 4 columns

In [6]:

    

from sklearn.feature_extraction import DictVectorizer


    

In [7]:

    

def explode(cues):
    return {}.fromkeys(cues.split('_'),True)


    

In [8]:

    

yes = data[data.Outcomes=='y']


    

In [9]:

    

vec = DictVectorizer(dtype=int,sparse=False)
M = vec.fit_transform([explode(c) for c in yes.Cues])
M


    

    
Out[9]:





array([[1, 1, 1],
       [0, 1, 1],
       [1, 1, 0],
       [0, 1, 0]])

In [10]:

    

P = np.diag(yes.Total/sum(yes.Total))
P


    

    
Out[10]:





array([[ 0.125,  0.   ,  0.   ,  0.   ],
       [ 0.   ,  0.375,  0.   ,  0.   ],
       [ 0.   ,  0.   ,  0.375,  0.   ],
       [ 0.   ,  0.   ,  0.   ,  0.125]])

In [11]:

    

left = M.T.dot(P).dot(M)
left


    

    
Out[11]:





array([[ 0.5  ,  0.5  ,  0.125],
       [ 0.5  ,  1.   ,  0.5  ],
       [ 0.125,  0.5  ,  0.5  ]])

In [12]:

    

O = yes.Frequency / yes.Total
O


    

    
Out[12]:





0    1.000000
2    0.666667
4    0.333333
6    0.000000
dtype: float64

In [13]:

    

right = M.T.dot(P).dot(O)
right


    

    
Out[13]:





array([ 0.25 ,  0.5  ,  0.375])

In [14]:

    

np.linalg.solve(left,right)


    

    
Out[14]:





array([ 0.33333333,  0.        ,  0.66666667])

In [15]:

    

vec.get_feature_names()


    

    
Out[15]:





['blue', 'pots', 'red']

In [16]:

    

A = pd.DataFrame({'Cue':vec.get_feature_names(),'Weight':np.linalg.solve(left,right)})
A


    

    
Out[16]:






  

    

      

      
Cue
      
Weight
    
  
  

    

      
0
      
 blue
      
 0.333333
    
    

      
1
      
 pots
      
 0.000000
    
    

      
2
      
  red
      
 0.666667
    
  

3 rows × 2 columns

In [17]:

    

from ndl import *


    

In [18]:

    

data = com.load_data('danks')
danks(data)


    

    
Out[18]:






  

    

      

      
y
      
n
    
  
  

    

      
blue
      
 0.333333
      
-0.333333
    
    

      
pots
      
 0.000000
      
 1.000000
    
    

      
red
      
 0.666667
      
-0.666667
    
  

3 rows × 2 columns

In [19]:

    

lexample = com.load_data('lexample')
lexample['Cues'] = orthoCoding(lexample['Word'],grams=1,sep='_')
lexample


    

    
Out[19]:






  

    

      

      
Word
      
Frequency
      
Outcomes
      
Cues
    
  
  

    

      
1 
      
  hand
      
  10
      
        hand
      
   h_a_n_d
    
    

      
2 
      
 hands
      
  20
      
 hand_plural
      
 h_a_n_d_s
    
    

      
3 
      
  land
      
   8
      
        land
      
   l_a_n_d
    
    

      
4 
      
 lands
      
   3
      
 land_plural
      
 l_a_n_d_s
    
    

      
5 
      
   and
      
  35
      
         and
      
     a_n_d
    
    

      
6 
      
   sad
      
  18
      
         sad
      
     s_a_d
    
    

      
7 
      
    as
      
  35
      
          as
      
       a_s
    
    

      
8 
      
   lad
      
 102
      
         lad
      
     l_a_d
    
    

      
9 
      
   lad
      
  54
      
  lad_plural
      
     l_a_d
    
    

      
10
      
  lass
      
 134
      
        lass
      
   l_a_s_s
    
  

10 rows × 4 columns

In [20]:

    

danks(lexample)


    

    




---------------------------------------------------------------------------
LinAlgError                               Traceback (most recent call last)
<ipython-input-20-8371b8e6c5ff> in <module>()
----> 1 danks(lexample)

/Users/malouf/Dropbox/learning/ndl.pyc in danks(data)
     61         right = MTP.dot(O)
     62 
---> 63         V = np.linalg.solve(left,right)
     64 
     65         result[outcome] = V

/Users/malouf/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/numpy/linalg/linalg.pyc in solve(a, b)
    379     signature = 'DD->D' if isComplexType(t) else 'dd->d'
    380     extobj = get_linalg_error_extobj(_raise_linalgerror_singular)
--> 381     r = gufunc(a, b, signature=signature, extobj=extobj)
    382 
    383     return wrap(r.astype(result_t))

/Users/malouf/Library/Enthought/Canopy_64bit/User/lib/python2.7/site-packages/numpy/linalg/linalg.pyc in _raise_linalgerror_singular(err, flag)
     88 
     89 def _raise_linalgerror_singular(err, flag):
---> 90     raise LinAlgError("Singular matrix")
     91 
     92 def _raise_linalgerror_nonposdef(err, flag):

LinAlgError: Singular matrix

In [21]:

    

ndl(lexample)


    

    
Out[21]:






  

    

      

      
and
      
as
      
hand
      
lad
      
land
      
lass
      
plural
      
sad
    
  
  

    

      
a
      
 0.375349
      
 1.034097
      
-4.884981e-15
      
 0.409446
      
-0.375349
      
-0.034097
      
-0.448823
      
-0.409446
    
    

      
d
      
-0.161751
      
-0.443564
      
 2.442491e-15
      
 0.394685
      
 0.161751
      
-0.556436
      
 0.531307
      
 0.605315
    
    

      
h
      
-0.688749
      
 0.053487
      
 1.000000e+00
      
 0.364738
      
-0.311251
      
-0.053487
      
 0.490994
      
-0.364738
    
    

      
l
      
-0.214565
      
-0.619968
      
 2.803313e-15
      
 0.165467
      
 0.214565
      
 0.619968
      
 0.222744
      
-0.165467
    
    

      
n
      
 0.611964
      
-0.420404
      
 7.771561e-16
      
-0.808440
      
 0.388036
      
 0.420404
      
-0.088284
      
-0.191560
    
    

      
s
      
-0.205220
      
-0.335424
      
 2.997602e-15
      
-0.540644
      
 0.205220
      
 0.335424
      
 0.272209
      
 0.540644
    
  

6 rows × 8 columns

In [ ]: